{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn')\n",
    "plt.gray()\n",
    "import mglearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_scaling()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "cancer = load_breast_cancer()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=1026)\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "scaler.fit(X_train)\n",
    "\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "print(X_train_scaled)\n",
    "print(X_test_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "# make synthetic data\n",
    "X, _ = make_blobs(n_samples=50, centers=5, random_state=4, cluster_std=2)\n",
    "# split it into training and test sets\n",
    "X_train, X_test = train_test_split(X, random_state=5, test_size=.1)\n",
    "# plot the training and test sets\n",
    "fig, axes = plt.subplots(1, 3, figsize=(13, 4))\n",
    "axes[0].scatter(X_train[:, 0], X_train[:, 1], c=mglearn.cm2(0), label=\"Training set\", s=60)\n",
    "axes[0].scatter(X_test[:, 0], X_test[:, 1], marker='^', c=mglearn.cm2(1), label=\"Test set\", s=60)\n",
    "axes[0].legend(loc='upper left')\n",
    "axes[0].set_title(\"Original Data\")\n",
    "\n",
    "# scale the data using MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "# visualize the properly scaled data\n",
    "axes[1].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1], c=mglearn.cm2(0), label=\"Training set\", s=60)\n",
    "axes[1].scatter(X_test_scaled[:, 0], X_test_scaled[:, 1], marker='^', c=mglearn.cm2(1), label=\"Test set\", s=60)\n",
    "axes[1].set_title(\"Scaled Data\")\n",
    "\n",
    "# rescale the test set separately\n",
    "# so test set min is 0 and test set max is 1\n",
    "# DO NOT DO THIS! For illustration purposes only.\n",
    "test_scaler = MinMaxScaler()\n",
    "test_scaler.fit(X_test)\n",
    "X_test_scaled_badly = test_scaler.transform(X_test)\n",
    "# visualize wrongly scaled data\n",
    "axes[2].scatter(X_train_scaled[:, 0], X_train_scaled[:, 1], c=mglearn.cm2(0), label=\"training set\", s=60)\n",
    "axes[2].scatter(X_test_scaled_badly[:, 0], X_test_scaled_badly[:, 1], marker='^', c=mglearn.cm2(1), label=\"test set\", s=60)\n",
    "axes[2].set_title(\"Improperly Scaled Data\")\n",
    "for ax in axes:\n",
    "    ax.set_xlabel(\"Feature 0\")\n",
    "    ax.set_ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=1026)\n",
    "svm = SVC(C=100)\n",
    "svm.fit(X_train, y_train)\n",
    "print('Test set accuracy:{:.2f}'.format(svm.score(X_test, y_test)))\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "svm.fit(X_train_scaled, y_train)\n",
    "print('Test set accuracy:{:.2f}'.format(svm.score(X_test_scaled, y_test)))\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "svm.fit(X_train_scaled, y_train)\n",
    "print('Test set accuracy:{:.2f}'.format(svm.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_pca_illustration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(15, 2, figsize=(10, 20))\n",
    "malignant = cancer.data[cancer.target == 0]\n",
    "benign = cancer.data[cancer.target == 1]\n",
    "\n",
    "ax = axes.ravel()\n",
    "\n",
    "for i in range(30):\n",
    "    _,bins=np.histogram(cancer.data[:,i],50)    \n",
    "    ax[i].hist(malignant[:,i], bins=bins, color=mglearn.cm3(0),alpha=0.5)    \n",
    "    ax[i].hist(benign[:, i], bins=bins, color=mglearn.cm3(2), alpha=.5)\n",
    "    ax[i].set_title(cancer.feature_names[i])\n",
    "    ax[i].set_yticks(())\n",
    "ax[0].set_xlabel(\"Feature magnitude\")\n",
    "ax[0].set_ylabel(\"Frequency\")\n",
    "ax[0].legend([\"malignant\", \"benign\"], loc=\"best\")\n",
    "fig.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(cancer.data)\n",
    "X_scaled = scaler.transform(cancer.data)\n",
    "from sklearn.decomposition import PCA\n",
    "# keep the first two principal components of the data\n",
    "pca = PCA(n_components=2)\n",
    "# fit PCA model to breast cancer data\n",
    "pca.fit(X_scaled)\n",
    "# transform data onto the first two principal components\n",
    "X_pca = pca.transform(X_scaled)\n",
    "print(\"Original shape: {}\".format(str(X_scaled.shape)))\n",
    "print(\"Reduced shape: {}\".format(str(X_pca.shape)))\n",
    "\n",
    "# plot first vs. second principal component, colored by class\n",
    "plt.figure(figsize=(8, 8))\n",
    "mglearn.discrete_scatter(X_pca[:, 0], X_pca[:, 1], cancer.target)\n",
    "plt.legend(cancer.target_names, loc=\"best\")\n",
    "plt.gca().set_aspect(\"equal\")\n",
    "plt.xlabel(\"First principal component\")\n",
    "plt.ylabel(\"Second principal component\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(pca.components_, cmap='viridis')\n",
    "plt.yticks([0, 1], [\"First component\", \"Second component\"])\n",
    "plt.colorbar()\n",
    "plt.xticks(range(len(cancer.feature_names)), cancer.feature_names, rotation=60, ha='left')\n",
    "plt.xlabel(\"Feature\")\n",
    "plt.ylabel(\"Principal components\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_lfw_people\n",
    "people = fetch_lfw_people(min_faces_per_person=20,resize=0.7, download_if_missing=False)\n",
    "image_shape = people.images[0].shape\n",
    "\n",
    "fix, axes = plt.subplots(2, 5, figsize=(15, 8), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "for target, image, ax in zip(people.target, people.images, axes.ravel()):\n",
    "    ax.imshow(image)\n",
    "    ax.set_title(people.target_names[target])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"people.images.shape: {}\".format(people.images.shape))\n",
    "print(\"Number of classes: {}\".format(len(people.target_names)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count how often each target appears\n",
    "counts = np.bincount(people.target)\n",
    "# print counts next to target names\n",
    "for i, (count, name) in enumerate(zip(counts, people.target_names)):\n",
    "    print(\"{0:25} {1:3}\".format(name, count), end=' ')\n",
    "    if (i + 1) % 3 == 0:\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = np.zeros(people.target.shape, dtype=bool)\n",
    "for target in np.unique(people.target):\n",
    "    mask[np.where(people.target==target)[0][:50]]=1\n",
    "X_people = people.data[mask]\n",
    "y_people = people.target[mask]\n",
    "X_people = X_people / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_people.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "# split the data into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_people, y_people, stratify=y_people, random_state=0)\n",
    "# build a KNeighborsClassifier using one neighbor\n",
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "knn.fit(X_train, y_train)\n",
    "print(\"Test set score of 1-nn: {:.2f}\".format(knn.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_pca_whitening()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=100, whiten=True, random_state=0).fit(X_train)\n",
    "X_train_pca = pca.transform(X_train)\n",
    "X_test_pca = pca.transform(X_test)\n",
    "print(\"X_train_pca.shape: {}\".format(X_train_pca.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "knn.fit(X_train_pca, y_train)\n",
    "print(\"Test set accuracy: {:.2f}\".format(knn.score(X_test_pca, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"pca.components_.shape: {}\".format(pca.components_.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fix, axes = plt.subplots(3, 5, figsize=(15, 12), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "for i, (component, ax) in enumerate(zip(pca.components_, axes.ravel())):\n",
    "    ax.imshow(component.reshape(image_shape),cmap='viridis')\n",
    "    ax.set_title(\"{}. component\".format((i + 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_nmf_illustration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "digits = load_digits()\n",
    "fig, axes = plt.subplots(2, 5, figsize=(10, 5), subplot_kw={'xticks':(), 'yticks': ()})\n",
    "for ax, img in zip(axes.ravel(), digits.images):\n",
    " ax.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build a PCA model\n",
    "pca = PCA(n_components=2)\n",
    "pca.fit(digits.data)\n",
    "# transform the digits data onto the first two principal components\n",
    "digits_pca = pca.transform(digits.data)\n",
    "colors = [\"#476A2A\", \"#7851B8\", \"#BD3430\", \"#4A2D4E\", \"#875525\", \"#A83683\", \"#4E655E\", \"#853541\", \"#3A3120\", \"#535D8E\"]\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.xlim(digits_pca[:, 0].min(), digits_pca[:, 0].max())\n",
    "plt.ylim(digits_pca[:, 1].min(), digits_pca[:, 1].max())\n",
    "for i in range(len(digits.data)):\n",
    "    # actually plot the digits as text instead of using scatter\n",
    "    plt.text(digits_pca[i, 0], digits_pca[i, 1], str(digits.target[i]), color = colors[digits.target[i]], fontdict={'weight': 'bold', 'size': 9})\n",
    "plt.xlabel(\"First principal component\")\n",
    "plt.ylabel(\"Second principal component\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "tsne = TSNE(random_state=1026)\n",
    "# use fit_transform instead of fit, as TSNE has no transform method\n",
    "digits_tsne = tsne.fit_transform(digits.data)\n",
    "\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.xlim(digits_tsne[:, 0].min(), digits_tsne[:, 0].max() + 1)\n",
    "plt.ylim(digits_tsne[:, 1].min(), digits_tsne[:, 1].max() + 1)\n",
    "for i in range(len(digits.data)):\n",
    "    # actually plot the digits as text instead of using scatter\n",
    "    plt.text(digits_tsne[i, 0], digits_tsne[i, 1], str(digits.target[i]), color = colors[digits.target[i]], fontdict={'weight': 'bold', 'size': 9})\n",
    "plt.xlabel(\"t-SNE feature 0\")\n",
    "plt.xlabel(\"t-SNE feature 1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-Means clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_kmeans_algorithm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_kmeans_boundaries()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.cluster import KMeans\n",
    "# generate synthetic two-dimensional data\n",
    "X, y = make_blobs(random_state=1)\n",
    "# build the clustering model\n",
    "kmeans = KMeans(n_clusters=3)\n",
    "kmeans.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], kmeans.labels_, markers='o')\n",
    "mglearn.discrete_scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], [0, 1, 2], markers='^', markeredgewidth=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(10, 5))\n",
    "# using two cluster centers:\n",
    "kmeans = KMeans(n_clusters=2)\n",
    "kmeans.fit(X)\n",
    "assignments = kmeans.labels_\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], assignments, ax=axes[0])\n",
    "# using five cluster centers:\n",
    "kmeans = KMeans(n_clusters=5)\n",
    "kmeans.fit(X)\n",
    "assignments = kmeans.labels_\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], assignments, ax=axes[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_varied, y_varied = make_blobs(n_samples=200, cluster_std=[1.0, 2.5, 0.5], random_state=170)\n",
    "y_pred = KMeans(n_clusters=3, random_state=0).fit_predict(X_varied)\n",
    "mglearn.discrete_scatter(X_varied[:, 0], X_varied[:, 1], y_pred)\n",
    "plt.legend([\"cluster 0\", \"cluster 1\", \"cluster 2\"], loc='best')\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate some random cluster data\n",
    "X, y = make_blobs(random_state=170, n_samples=600)\n",
    "rng = np.random.RandomState(74)\n",
    "# transform the data to be stretched\n",
    "transformation = rng.normal(size=(2, 2))\n",
    "X = np.dot(X, transformation)\n",
    "# cluster the data into three clusters\n",
    "kmeans = KMeans(n_clusters=3)\n",
    "kmeans.fit(X)\n",
    "y_pred = kmeans.predict(X)\n",
    "# plot the cluster assignments and cluster centers\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y_pred, cmap=mglearn.cm3)\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], marker='^', c=[0, 1, 2], s=100, linewidth=2, cmap=mglearn.cm3)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate synthetic two_moons data (with less noise this time)\n",
    "from sklearn.datasets import make_moons\n",
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "# cluster the data into two clusters\n",
    "kmeans = KMeans(n_clusters=2)\n",
    "kmeans.fit(X)\n",
    "y_pred = kmeans.predict(X)\n",
    "# plot the cluster assignments and cluster centers\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y_pred, cmap=mglearn.cm2, s=60)\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1],\n",
    " marker='^', c=[mglearn.cm2(0), mglearn.cm2(1)], s=100, linewidth=2)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import NMF\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_people, y_people, stratify=y_people, random_state=0)\n",
    "nmf = NMF(n_components=100, random_state=0)\n",
    "nmf.fit(X_train)\n",
    "pca = PCA(n_components=100, random_state=0)\n",
    "pca.fit(X_train)\n",
    "kmeans = KMeans(n_clusters=100, random_state=0)\n",
    "kmeans.fit(X_train)\n",
    "X_reconstructed_pca = pca.inverse_transform(pca.transform(X_test))\n",
    "X_reconstructed_kmeans = kmeans.cluster_centers_[kmeans.predict(X_test)]\n",
    "X_reconstructed_nmf = np.dot(nmf.transform(X_test), nmf.components_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3, 5, figsize=(8, 8), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "fig.suptitle(\"Extracted Components\")\n",
    "for ax, comp_kmeans, comp_pca, comp_nmf in zip(axes.T, kmeans.cluster_centers_, pca.components_, nmf.components_):\n",
    "    ax[0].imshow(comp_kmeans.reshape(image_shape))\n",
    "    ax[1].imshow(comp_pca.reshape(image_shape), cmap='viridis')\n",
    "    ax[2].imshow(comp_nmf.reshape(image_shape))\n",
    "axes[0, 0].set_ylabel(\"kmeans\")\n",
    "axes[1, 0].set_ylabel(\"pca\")\n",
    "axes[2, 0].set_ylabel(\"nmf\")\n",
    "fig, axes = plt.subplots(4, 5, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(8, 8))\n",
    "fig.suptitle(\"Reconstructions\")\n",
    "for ax, orig, rec_kmeans, rec_pca, rec_nmf in zip(axes.T, X_test, X_reconstructed_kmeans, X_reconstructed_pca,X_reconstructed_nmf):\n",
    "    ax[0].imshow(orig.reshape(image_shape))\n",
    "    ax[1].imshow(rec_kmeans.reshape(image_shape))\n",
    "    ax[2].imshow(rec_pca.reshape(image_shape))\n",
    "    ax[3].imshow(rec_nmf.reshape(image_shape))\n",
    "axes[0, 0].set_ylabel(\"original\")\n",
    "axes[1, 0].set_ylabel(\"kmeans\")\n",
    "axes[2, 0].set_ylabel(\"pca\")\n",
    "axes[3, 0].set_ylabel(\"nmf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "kmeans = KMeans(n_clusters=10, random_state=0)\n",
    "kmeans.fit(X)\n",
    "y_pred = kmeans.predict(X)\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y_pred, s=60, cmap='Paired')\n",
    "plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s=60,marker='^', c=range(kmeans.n_clusters), linewidth=2, cmap='Paired')\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")\n",
    "print(\"Cluster memberships:\\n{}\".format(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distance_features = kmeans.transform(X)\n",
    "print(\"Distance feature shape: {}\".format(distance_features.shape))\n",
    "print(\"Distance features:\\n{}\".format(distance_features))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 凝聚聚类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_agglomerative_algorithm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "X,y = make_blobs(random_state=1026)\n",
    "agg = AgglomerativeClustering(n_clusters =3)\n",
    "assignment = agg.fit_predict(X)\n",
    "\n",
    "mglearn.discrete_scatter(X[:,0],X[:,1],assignment)\n",
    "plt.xlabel('Feature 0')\n",
    "plt.ylabel('Feature 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_agglomerative()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.cluster.hierarchy import dendrogram, ward\n",
    "\n",
    "X,y = make_blobs(random_state=1026, n_samples=12)\n",
    "linkage_array = ward(X)\n",
    "dendrogram(linkage_array)\n",
    "\n",
    "# Mark the cuts in the tree that signify two or three clusters\n",
    "ax = plt.gca()\n",
    "bounds = ax.get_xbound()\n",
    "ax.plot(bounds, [7.25, 7.25], '--', c='k')\n",
    "ax.plot(bounds, [4, 4], '--', c='k')\n",
    "ax.text(bounds[1], 7.25, ' two clusters', va='center', fontdict={'size': 15})\n",
    "ax.text(bounds[1], 4, ' three clusters', va='center', fontdict={'size': 15})\n",
    "plt.xlabel(\"Sample index\")\n",
    "plt.ylabel(\"Cluster distance\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DBSCAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import DBSCAN\n",
    "X,y = make_blobs(random_state=1026, n_samples=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbscan = DBSCAN()\n",
    "clusters  = dbscan.fit_predict(X)\n",
    "print('Cluster membership:\\n{}'.format(clusters))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_dbscan()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "# rescale the data to zero mean and unit variance\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X_scaled = scaler.transform(X)\n",
    "dbscan = DBSCAN(eps=0.3)\n",
    "clusters = dbscan.fit_predict(X_scaled)\n",
    "# plot the cluster assignments\n",
    "plt.scatter(X_scaled[:, 0], X_scaled[:, 1], c=clusters, cmap=mglearn.cm2, s=60)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing and Evaluating Clustering Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.cluster import adjusted_rand_score\n",
    "X, y = make_moons(n_samples=200, noise=0.05, random_state=0)\n",
    "# rescale the data to zero mean and unit variance\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X_scaled = scaler.transform(X)\n",
    "fig, axes = plt.subplots(1, 4, figsize=(15, 3), subplot_kw={'xticks': (), 'yticks': ()})\n",
    "# make a list of algorithms to use\n",
    "algorithms = [KMeans(n_clusters=2), AgglomerativeClustering(n_clusters=2), DBSCAN()]\n",
    "# create a random cluster assignment for reference\n",
    "random_state = np.random.RandomState(seed=0)\n",
    "random_clusters = random_state.randint(low=0, high=2, size=len(X))\n",
    "# plot random assignment\n",
    "axes[0].scatter(X_scaled[:, 0], X_scaled[:, 1], c=random_clusters, cmap=mglearn.cm3, s=60)\n",
    "axes[0].set_title(\"Random assignment - ARI: {:.2f}\".format(adjusted_rand_score(y, random_clusters)))\n",
    "for ax, algorithm in zip(axes[1:], algorithms):\n",
    "    # plot the cluster assignments and cluster centers\n",
    "    clusters = algorithm.fit_predict(X_scaled)\n",
    "    ax.scatter(X_scaled[:, 0], X_scaled[:, 1], c=clusters,cmap=mglearn.cm3, s=60)\n",
    "    ax.set_title(\"{} - ARI: {:.2f}\".format(algorithm.__class__.__name__, adjusted_rand_score(y, clusters)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics.cluster import normalized_mutual_info_score\n",
    "# these two labelings of points correspond to the same clustering\n",
    "clusters1 = [0, 0, 1, 1, 0]\n",
    "clusters2 = [1, 1, 0, 0, 1]\n",
    "# accuracy is zero, as none of the labels are the same\n",
    "print(\"Accuracy: {:.2f}\".format(accuracy_score(clusters1, clusters2)))\n",
    "# adjusted rand score is 1, as the clustering is exactly the same\n",
    "print(\"ARI: {:.2f}\".format(adjusted_rand_score(clusters1, clusters2)))\n",
    "print(\"NMI: {:.2f}\".format(normalized_mutual_info_score(clusters1, clusters2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.cluster import silhouette_score\n",
    "X, y = make_moons(n_samples=300, noise =0.05, random_state=1026)\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X_scaled = scaler.transform(X)\n",
    "\n",
    "fig, axes = plt.subplots(1,4,figsize=(15,3), subplot_kw={'xticks':(), 'yticks':()})\n",
    "\n",
    "rng = np.random.default_rng()\n",
    "random_clusters = rng.integers(0, 2, size=len(X))\n",
    "\n",
    "axes[0].scatter(X_scaled[:,0],X_scaled[:,1], c=random_clusters, cmap=mglearn.cm3, s=60)\n",
    "axes[0].set_title('Random assignment:{:.3f}'.format(silhouette_score(X_scaled, random_clusters)))\n",
    "\n",
    "algorithms = [KMeans(n_clusters=2), AgglomerativeClustering(n_clusters=2), DBSCAN()]\n",
    "\n",
    "for ax, algorithm in zip(axes[1:], algorithms):\n",
    "    clusters = algorithm.fit_predict(X_scaled)\n",
    "    ax.scatter(X_scaled[:,0],X_scaled[:,1], c=clusters, cmap=mglearn.cm3, s=60)\n",
    "    ax.set_title('{} : {:.3f}'.format(algorithms.__class__.__name__, silhouette_score(X_scaled, clusters)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=100, whiten = True, random_state=1026)\n",
    "pca.fit_transform(X_people)\n",
    "X_pca = pca.transform(X_people)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbscan = DBSCAN()\n",
    "labels = dbscan.fit_predict(X_pca)\n",
    "print('unique labels:{}'.format(np.unique(labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbscan = DBSCAN(min_samples=3)\n",
    "labels = dbscan.fit_predict(X_pca)\n",
    "print(\"Unique labels: {}\".format(np.unique(labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dbscan = DBSCAN(min_samples=3, eps=15)\n",
    "labels = dbscan.fit_predict(X_pca)\n",
    "print(\"Unique labels: {}\".format(np.unique(labels)))\n",
    "print('Number of points per cluster: {}'.format(np.bincount(labels+1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "noise = X_people[labels==-1]\n",
    "fig, axes = plt.subplots(2, 7, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(12, 4))\n",
    "for image, ax in zip(noise, axes.ravel()):\n",
    "    ax.imshow(image.reshape(image_shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for eps in [1,3,5,7,9,11,13,15]:\n",
    "    print('\\n eps={}'.format(eps))\n",
    "    dbscan = DBSCAN(eps=eps, min_samples=3)\n",
    "    labels = dbscan.fit_predict(X_pca)\n",
    "    print('Cluster present:{}'.format(np.unique(labels)))\n",
    "    print('Cluster sizes: {}'.format(np.bincount(labels+1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=10, random_state=0)\n",
    "labels_km  = km.fit_predict(X_pca)\n",
    "print('Cluster sizes k-means:{}'.format(np.bincount(labels_km)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 5, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(12, 4))\n",
    "for center, ax in zip(km.cluster_centers_, axes.ravel()):\n",
    "    ax.imshow(pca.inverse_transform(center).reshape(image_shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_kmeans_faces(km, pca, X_pca, X_people, y_people, people.target_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract clusters with ward agglomerative clustering\n",
    "agglomerative = AgglomerativeClustering(n_clusters=10)\n",
    "labels_agg = agglomerative.fit_predict(X_pca)\n",
    "print(\"Cluster sizes agglomerative clustering: {}\".format( np.bincount(labels_agg)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"ARI: {:.2f}\".format(adjusted_rand_score(labels_agg, labels_km)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linkage_array = ward(X_pca)\n",
    "# now we plot the dendrogram for the linkage_array\n",
    "# containing the distances between clusters\n",
    "plt.figure(figsize=(20, 5))\n",
    "dendrogram(linkage_array, p=7, truncate_mode='level', no_labels=True)\n",
    "plt.xlabel(\"Sample index\")\n",
    "plt.ylabel(\"Cluster distance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_clusters = 10\n",
    "for cluster in range(n_clusters):\n",
    "    mask = labels_agg == cluster\n",
    "    fig, axes = plt.subplots(1, 10, subplot_kw={'xticks': (), 'yticks': ()},figsize=(15, 8))\n",
    "    axes[0].set_ylabel(np.sum(mask))\n",
    "    for image, label, asdf, ax in zip(X_people[mask], y_people[mask], labels_agg[mask], axes):\n",
    "        ax.imshow(image.reshape(image_shape))\n",
    "        ax.set_title(people.target_names[label].split()[-1],fontdict={'fontsize': 9})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract clusters with ward agglomerative clustering\n",
    "agglomerative = AgglomerativeClustering(n_clusters=40)\n",
    "labels_agg = agglomerative.fit_predict(X_pca)\n",
    "print(\"cluster sizes agglomerative clustering: {}\".format(np.bincount(labels_agg)))\n",
    "n_clusters = 40\n",
    "for cluster in [10, 13, 19, 22, 36]: # hand-picked \"interesting\" clusters\n",
    "    mask = labels_agg == cluster\n",
    "    fig, axes = plt.subplots(1, 15, subplot_kw={'xticks': (), 'yticks': ()}, figsize=(15, 8))\n",
    "    cluster_size = np.sum(mask)\n",
    "    axes[0].set_ylabel(\"#{}: {}\".format(cluster, cluster_size))\n",
    "    for image, label, asdf, ax in zip(X_people[mask], y_people[mask], labels_agg[mask], axes):\n",
    "        ax.imshow(image.reshape(image_shape))\n",
    "        ax.set_title(people.target_names[label].split()[-1],fontdict={'fontsize': 9})\n",
    "    for i in range(cluster_size, 15):\n",
    "        axes[i].set_visible(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e21935a792cabaadabb39e077efde3aadff814271e124eae6fb74a8aae0f5081"
  },
  "kernelspec": {
   "display_name": "Python 3.10.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
